Skip to content

Commit 179d43c

Browse files
LeoMcALeoMcA
committed
move build pipeline to circleci
1 parent b8cb682 commit 179d43c

File tree

14 files changed

+131
-198
lines changed

14 files changed

+131
-198
lines changed

‎.circleci/config.yml

Lines changed: 31 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
version: 2
22
jobs:
3-
build:
3+
lint:
44
docker:
55
- image: circleci/python:3.9-buster
66
steps:
@@ -10,15 +10,19 @@ jobs:
1010
command: |
1111
sudo pip install --upgrade pre-commit==2.15.0
1212
pre-commit run --all-files
13+
test-and-build:
14+
docker:
15+
- image: circleci/python:3.9-buster
16+
steps:
17+
- checkout
1318
- setup_remote_docker:
14-
version: 17.10.0-ce
19+
version: 20.10.7
1520
docker_layer_caching: true
1621
- run:
17-
name: Build docker images
18-
command: make build-ci
19-
- run:
20-
name: Run mocha tests
21-
command: make test-js-ci
22+
name: Build test image
23+
command: |
24+
cp .env-build .env
25+
DOCKER_BUILDKIT=1 ./bin/dc.sh build --progress=plain test
2226
- run:
2327
# copy synonym files to elasticsearch7 container, since circleci doesn't support volume mounts:
2428
# https://circleci.com/docs/2.0/building-docker-images/#mounting-folders
@@ -28,4 +32,23 @@ jobs:
2832
docker cp ./kitsune/search/dictionaries/synonyms/. project_elasticsearch7_1:/usr/share/elasticsearch/config/synonyms
2933
- run:
3034
name: Run unit tests
31-
command: make test-ci
35+
command: ./bin/dc.sh run test ./bin/run-unit-tests.sh
36+
- run:
37+
name: Run js tests
38+
command: ./bin/dc.sh run test ./bin/run-mocha-tests.sh
39+
- run:
40+
name: Build prod image
41+
command: DOCKER_BUILDKIT=1 ./bin/dc.sh build --progress=plain prod
42+
- run:
43+
name: Push prod image
44+
command: |
45+
docker login -u "${DOCKER_USERNAME}" -p "${DOCKER_PASSWORD}"
46+
source docker/bin/set_git_env_vars.sh
47+
# docker push mozilla/kitsune:prod-${GIT_COMMIT_SHORT}
48+
docker logout
49+
workflows:
50+
version: 2
51+
lint-test-build:
52+
jobs:
53+
- lint
54+
- test-and-build

‎.env-build

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,5 @@ CELERY_TASK_ALWAYS_EAGER=True
77
CSRF_COOKIE_SECURE=False
88
DATABASE_URL=sqlite://
99
DATABASE_READ_ONLY_URL=sqlite://
10-
ES7_URLS=elasticsearch:9200
1110
SESSION_COOKIE_SECURE=False
1211
SECRET_KEY=secret

‎.env-test

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
TEST=True
12
DEBUG=False
23
# Some cron jobs are skipped on stage.
34
STAGE=False
@@ -16,7 +17,6 @@ CACHE_URL=redis://redis:6379/3
1617
CSRF_COOKIE_SECURE=False
1718
DATABASE_URL=mysql://root:kitsune@mariadb:3306/kitsune
1819
DB_CONN_MAX_AGE=0
19-
ES_URLS=elasticsearch:9200
2020
SECRET_KEY=secret
2121
REUSE_STATIC=1
2222
REUSE_DB=0

‎Dockerfile

Lines changed: 65 additions & 75 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,6 @@
1-
################################
2-
# Frontend dependencies builder
3-
#
4-
FROM node:12 AS frontend-base
5-
6-
WORKDIR /app
7-
COPY ["./package.json", "./package-lock.json", "prepare_django_assets.js", "/app/"]
8-
COPY ./kitsune/sumo/static/sumo /app/kitsune/sumo/static/sumo
9-
RUN npm run production
10-
11-
################################
12-
# Python dependencies builder
13-
#
1+
#######################
2+
# Common dependencies #
3+
#######################
144
FROM python:3.9-buster AS base
155

166
WORKDIR /app
@@ -26,83 +16,90 @@ RUN python -m venv /venv
2616
RUN pip install --upgrade "pip==21.0.1"
2717
RUN useradd -d /app -M --uid 1000 --shell /usr/sbin/nologin kitsune
2818

19+
RUN apt-get update && apt-get install apt-transport-https && \
20+
curl -sL https://deb.nodesource.com/setup_12.x | bash -
2921
RUN apt-get update && \
3022
apt-get install -y --no-install-recommends \
3123
gettext build-essential \
3224
libxml2-dev libxslt1-dev zlib1g-dev git \
3325
libjpeg-dev libffi-dev libssl-dev libxslt1.1 \
34-
libmariadb3 mariadb-client && \
26+
libmariadb3 mariadb-client \
27+
optipng nodejs zip && \
3528
rm -rf /var/lib/apt/lists/*
3629

3730
COPY ./requirements/*.txt /app/requirements/
38-
39-
RUN pip install --no-cache-dir --require-hashes -r requirements/default.txt && \
40-
pip install --no-cache-dir --require-hashes -r requirements/dev.txt
41-
42-
ARG GIT_SHA=head
43-
ENV GIT_SHA=${GIT_SHA}
31+
RUN pip install --no-cache-dir --require-hashes -r requirements/default.txt
4432

4533

46-
################################
47-
# Developer image
48-
#
49-
FROM base AS base-dev
50-
RUN apt-get update && apt-get install apt-transport-https && \
51-
curl -sL https://deb.nodesource.com/setup_12.x | bash -
52-
RUN apt-get update && apt-get install -y --no-install-recommends optipng nodejs zip && \
53-
rm -rf /var/lib/apt/lists/*
34+
#####################
35+
# Development image #
36+
#####################
37+
FROM base AS dev
5438

39+
RUN pip install --no-cache-dir --require-hashes -r requirements/dev.txt
5540

56-
################################
57-
# Fetch locales
58-
#
59-
FROM python:3.9-buster AS locales
6041

61-
WORKDIR /app
42+
#########################
43+
# Frontend dependencies #
44+
#########################
45+
FROM base AS base-frontend
6246

63-
RUN apt-get update && \
64-
apt-get install -y --no-install-recommends gettext
47+
COPY ./package*.json /app/
48+
RUN npm run install-prod && npm run copy:protocol
6549

66-
ENV PATH="/venv/bin:$PATH"
50+
COPY prepare_django_assets.js .
51+
RUN npm run postinstall
6752

68-
COPY --from=base /venv /venv
53+
COPY kitsune/sumo/static/sumo/scss kitsune/sumo/static/sumo/scss
54+
RUN npm run build:scss && npm run build:postcss
6955

7056
COPY . .
57+
RUN cp .env-build .env && \
58+
./manage.py nunjucks_precompile
7159

72-
ARG LOCALE_ENV=master
73-
ENV LOCALE_ENV=${LOCALE_ENV}
74-
RUN ./docker/bin/fetch-l10n-files.sh
75-
RUN ./scripts/compile-linted-mo.sh && \
76-
find ./locale ! -name '*.mo' -type f -delete
7760

78-
ARG GIT_SHA=head
79-
ENV GIT_SHA ${GIT_SHA}
61+
########################
62+
# Testing dependencies #
63+
########################
64+
FROM base AS test-deps
8065

66+
RUN pip install --no-cache-dir --require-hashes -r requirements/test.txt
8167

82-
################################
83-
# Staticfiles builder
84-
#
85-
FROM base-dev AS staticfiles
8668

87-
COPY --from=frontend-base --chown=kitsune:kitsune /app/assets /app/assets
88-
COPY --from=frontend-base --chown=kitsune:kitsune /app/node_modules /app/node_modules
89-
COPY --from=locales /app/locale /app/locale
69+
#################
70+
# Testing image #
71+
#################
72+
FROM base-frontend AS test
9073

91-
COPY . .
74+
COPY --from=test-deps /venv /venv
9275

93-
RUN cp .env-build .env && \
94-
./manage.py nunjucks_precompile && \
76+
RUN cp .env-test .env && \
9577
./manage.py compilejsi18n && \
78+
./manage.py collectstatic --noinput
79+
80+
81+
##########################
82+
# Production dependences #
83+
##########################
84+
FROM base-frontend AS prod-deps
85+
86+
ARG LOCALE_ENV=master
87+
ENV LOCALE_ENV=${LOCALE_ENV}
88+
RUN ./docker/bin/fetch-l10n-files.sh
89+
RUN ./scripts/compile-linted-mo.sh && \
90+
find ./locale ! -name '*.mo' -type f -delete
91+
92+
RUN ./manage.py compilejsi18n && \
9693
# minify jsi18n files:
9794
find jsi18n/ -name "*.js" -exec sh -c 'npx uglifyjs "$1" -o "${1%.js}-min.js"' sh {} \; && \
9895
./manage.py collectstatic --noinput && \
9996
npx svgo -r -f static
10097

10198

102-
################################
103-
# Full prod image sans locales
104-
#
105-
FROM python:3.9-slim-buster AS full-no-locales
99+
##########################
100+
# Clean production image #
101+
##########################
102+
FROM python:3.9-slim-buster AS prod
106103

107104
WORKDIR /app
108105

@@ -113,32 +110,25 @@ ENV LANG=C.UTF-8
113110
ENV PYTHONDONTWRITEBYTECODE=1
114111
ENV PYTHONUNBUFFERED=1
115112

113+
RUN groupadd --gid 1000 kitsune && useradd -g kitsune --uid 1000 --shell /usr/sbin/nologin kitsune
114+
115+
COPY --from=prod-deps --chown=kitsune:kitsune /venv /venv
116+
COPY --from=prod-deps --chown=kitsune:kitsune /app/locale /app/locale
117+
COPY --from=prod-deps --chown=kitsune:kitsune /app/static /app/static
118+
119+
COPY --chown=kitsune:kitsune . .
120+
121+
# apt-get after copying everything to ensure we're always getting the latest packages in the prod image
116122
RUN apt-get update && \
123+
apt-get upgrade && \
117124
apt-get install -y --no-install-recommends \
118125
libmariadb3 optipng mariadb-client \
119126
libxslt1.1 && \
120127
rm -rf /var/lib/apt/lists/*
121128

122-
RUN groupadd --gid 1000 kitsune && useradd -g kitsune --uid 1000 --shell /usr/sbin/nologin kitsune
123-
124-
COPY --from=base --chown=kitsune:kitsune /venv /venv
125-
COPY --from=staticfiles --chown=kitsune:kitsune /app/static /app/static
126-
127-
COPY --chown=kitsune:kitsune . .
128-
129129
RUN mkdir /app/media && chown kitsune:kitsune /app/media
130130

131131
USER kitsune
132132

133133
ARG GIT_SHA=head
134134
ENV GIT_SHA ${GIT_SHA}
135-
136-
137-
################################
138-
# Full final prod image
139-
#
140-
FROM full-no-locales AS full
141-
142-
USER root
143-
COPY --from=locales --chown=kitsune:kitsune /app/locale /app/locale
144-
USER kitsune

‎bin/dc.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,4 +3,4 @@
33
source docker/bin/set_git_env_vars.sh
44
export LOCALE_ENV=production
55

6-
docker-compose "$@"
6+
docker-compose -f docker-compose.yml -f docker-compose.ci.yml "$@"

‎docker-compose.ci.yml

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
version: "3.4"
2+
services:
3+
test:
4+
build:
5+
context: .
6+
target: test
7+
env_file: .env-test
8+
depends_on:
9+
- mariadb
10+
- elasticsearch7
11+
- redis
12+
13+
prod:
14+
build:
15+
context: .
16+
target: prod
17+
args:
18+
- GIT_SHA
19+
- LOCALE_ENV
20+
image: mozilla/kitsune:prod-${GIT_COMMIT_SHORT:-latest}

0 commit comments

Comments
 (0)